from langchain_community.document_loaders import WebBaseLoader
from urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
import requests

# 创建自定义会话
session = requests.Session()

# 配置重试策略
retry_strategy = Retry(
    total=3,
    backoff_factor=1,
    status_forcelist=[429, 500, 502, 503, 504],
    allowed_methods=["GET"]
)

# 配置适配器
adapter = HTTPAdapter(max_retries=retry_strategy)
session.mount("http://", adapter)
session.mount("https://", adapter)

# 使用自定义会话的加载器
loader = WebBaseLoader(
    web_paths=["https://baike.baidu.com/item/%E4%BB%BB%E5%8A%A1%E5%88%86%E8%A7%A3%E6%B3%95"],
    requests_per_second=1,  # 限制请求频率
    requests_kwargs={
        "timeout": 30,
        "headers": {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
        }
    }
)

# 替换加载器的会话
loader.session = session

try:
    documents = loader.load()
    print(f"成功加载 {len(documents)} 个文档")
except Exception as e:
    print(f"加载失败: {e}")